﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.IO;
using System.Text;

using Iveonik.Stemmers;

namespace Reckoning
{
    /// <summary>
    /// Default implementation of count filter
    /// </summary>
    public class CountFilter : ICountFilter
    {
        #region Class StemInfo
        /// <summary>
        /// Helper class
        /// </summary>
        private class StemInfo
        {
            /// <summary>
            /// The word that will be used to represent all words that have this stem
            /// </summary>
            public string Word
            {
                get
                {
                    uint minLen = uint.MaxValue;
                    string word = string.Empty;

                    // choose the word that has the shortest length to represent them all
                    foreach (KeyValuePair<string, uint> pair in _allWordCounts)
                    {
                        if (pair.Key.Length < minLen || (pair.Key.Length == minLen && pair.Key.CompareTo(word) == 1))
                        {
                            minLen = (uint) pair.Key.Length;
                            word = pair.Key;
                        }
                    }

                    return word;
                }
            }

            /// <summary>
            /// The total count of words that have this stem
            /// </summary>
            public uint Count
            {
                get
                {
                    uint total = 0;
                    foreach (KeyValuePair<string, uint> pair in _allWordCounts)
                    {
                        total += pair.Value;
                    }

                    return total;
                }
            }

            /// <summary>
            /// The words that have this stem and their counts
            /// </summary>
            private Dictionary<string, uint> _allWordCounts = new Dictionary<string, uint>();
            public Dictionary<string, uint> AllWordCounts
            {
                get
                {
                    return _allWordCounts;
                }
            }

            /// <summary>
            /// Constructor
            /// </summary>
            public StemInfo()
            {
            }

            /// <summary>
            /// Add a word that has this stem
            /// </summary>
            /// <param name="word">the word</param>
            /// <param name="count">count of the word</param>
            public void Add(string word, uint count)
            {
                _allWordCounts.Add(word, count);
            }
        }
        #endregion

        #region ICountFilter Properties
        public void Run(CountOptions countOpt,
                        Dictionary<string, uint> rawCounts,
                        uint totalWordCount,
                        out List<WordCountInfo> finalCounts,
                        out bool referenceDataOK)
        {
            finalCounts = new List<WordCountInfo>();
            referenceDataOK = false;

            // process the reference data            
            Dictionary<string, double> referenceData = new Dictionary<string, double>();
            if (countOpt.UseReferenceFile)
                referenceDataOK = ProcessReference(countOpt, ref referenceData);            

            List<WordCountInfo> filteredCounts = new List<WordCountInfo>();

            // now process the input
            if (countOpt.DoStemming)
                Stemming(countOpt, totalWordCount, rawCounts, referenceData, ref filteredCounts);
            else
                Counting(countOpt, totalWordCount, rawCounts, referenceData, ref filteredCounts);

            RankAndSort(ref filteredCounts);

            Threshold(countOpt, totalWordCount, ref filteredCounts, ref finalCounts);
        }
        #endregion

        #region Private Methods
        private void Threshold(CountOptions countOpt,
                               uint totalWordCount,
                               ref List<WordCountInfo> filteredCounts,
                               ref List<WordCountInfo> finalCounts)
        {
            uint percentThresh = 100;
            uint topNThresh = (uint) filteredCounts.Count;

            if (countOpt.PercentThresh < percentThresh)
                percentThresh = countOpt.PercentThresh;

            if (countOpt.TopNThresh < topNThresh)
                topNThresh = countOpt.TopNThresh;

            // find the cutoff count value for percent threshold
            uint runningThresh = (uint) Math.Round(((double) (percentThresh * totalWordCount)) / 100.0);
            uint running = 0;
            uint threshVal = 0;
            foreach (WordCountInfo wci in filteredCounts)
            {
                running += wci.Count;
                if (running >= runningThresh)
                {
                    threshVal = wci.Count;
                    break;
                }
            }

            // get all the terms that survive the threshold(s) 
            // always add words that tie with the word at the threshold
            uint last = uint.MaxValue;
            foreach (WordCountInfo wci in filteredCounts)
            {
                if (wci.Count < threshVal || (finalCounts.Count >= topNThresh && wci.Count < last))
                    break;
                
                finalCounts.Add(wci);
                last = wci.Count;
            }
        }

        /// <summary>
        /// Determine whether a word is a stop word
        /// </summary>
        /// <param name="word">the word to check for</param>
        private bool IsStopWord(CountOptions countOpt, string word)
        {
            if (countOpt.StopWords == null)
                return false;

            foreach (StopWordInfo swi in countOpt.StopWords)
            {
                if (swi.Word == word)
                    return true;
            }

            return false;
        }

        /// <summary>
        /// Process the reference raw counts
        /// </summary>
        /// <param name="countOpt">count options</param>        
        /// <param name="referenceData">reference data after processing</param>        
        /// <returns>true if reference data was read</returns>
        private bool ProcessReference(CountOptions countOpt,
                                      ref Dictionary<string, double> referenceData)
        {
            bool ret = false;
            referenceData.Clear();
            EnglishStemmer stemmer = new EnglishStemmer();
            Dictionary<string, StemInfo> stems = new Dictionary<string, StemInfo>();

            ReferenceDataFile refFile = new ReferenceDataFile();
            Dictionary<string, uint> rawReferenceData;
            uint totalReferenceCount;
            string version;
            ret = refFile.Read(countOpt.ReferenceFilePath, out rawReferenceData, out totalReferenceCount, out version);

            // put every word under its stem            
            foreach (KeyValuePair<string, uint> pair in rawReferenceData)
            {
                // ignore stop words
                if (countOpt.UseStopWords && IsStopWord(countOpt, pair.Key))
                    continue;

                string s;
                if (countOpt.DoStemming)
                    s = stemmer.Stem(pair.Key);
                else
                    s = pair.Key;

                if (!stems.ContainsKey(s))
                    stems.Add(s, new StemInfo());

                stems[s].Add(pair.Key, pair.Value);
            }

            // convert to count + stem list                        
            foreach (KeyValuePair<string, StemInfo> pair in stems)
                referenceData[pair.Value.Word] = ((double) pair.Value.Count) / totalReferenceCount * 100;

            return ret;
        }

        /// <summary>
        /// Do stemming and populate results
        /// </summary>
        private void Stemming(CountOptions countOpt,
                              uint totalWordCount,
                              Dictionary<string, uint> rawCounts,
                              Dictionary<string, double> referenceData,
                              ref List<WordCountInfo> filteredCounts)
        {
            EnglishStemmer stemmer = new EnglishStemmer();
            Dictionary<string, StemInfo> stems = new Dictionary<string, StemInfo>();

            // put every word under its stem
            foreach (KeyValuePair<string, uint> pair in rawCounts)
            {
                // ignore stop words
                if (countOpt.UseStopWords && IsStopWord(countOpt, pair.Key))
                    continue;

                string s = stemmer.Stem(pair.Key);

                if (!stems.ContainsKey(s))
                    stems.Add(s, new StemInfo());

                stems[s].Add(pair.Key, pair.Value);
            }

            // convert to count + stem list
            filteredCounts.Clear();
            foreach (KeyValuePair<string, StemInfo> pair in stems)
            {
                StemInfo si = (StemInfo) pair.Value;
                StringBuilder sb = new StringBuilder();

                double refPot = 0.0;
                if (referenceData != null && referenceData.ContainsKey(si.Word))
                    refPot = referenceData[si.Word];

                filteredCounts.Add(new WordCountInfo(si.Word, si.AllWordCounts, si.Count, ((double) si.Count) / totalWordCount * 100, refPot));
            }
        }

        /// <summary>
        /// Just count the words
        /// </summary>
        private void Counting(CountOptions countOpt,
                              uint totalWordCount,
                              Dictionary<string, uint> rawCounts,
                              Dictionary<string, double> referenceData,
                              ref List<WordCountInfo> filteredCounts)
        {
            // add all the words            
            foreach (KeyValuePair<string, uint> pair in rawCounts)
            {
                // ignore stop words
                if (countOpt.UseStopWords && IsStopWord(countOpt, pair.Key))
                    continue;

                double refPot = 0.0;
                if (referenceData != null && referenceData.ContainsKey(pair.Key))
                    refPot = referenceData[pair.Key];

                filteredCounts.Add(new WordCountInfo(pair.Key, null, pair.Value, ((double) pair.Value) / totalWordCount * 100, refPot));
            }
        }

        private void RankAndSort(ref List<WordCountInfo> filteredCounts)
        {
            // sort the list
            filteredCounts.Sort(new WordCountInfoComparer());

            // set the ranks
            if (filteredCounts.Count > 0)
                filteredCounts[0].Rank = 1;
            for (int i = 1; i < filteredCounts.Count; ++i)
            {
                if (filteredCounts[i] == filteredCounts[i - 1])
                    filteredCounts[i].Rank = filteredCounts[i - 1].Rank;
                else
                    filteredCounts[i].Rank = i + 1;
            }
        }
        #endregion
    }
}
